{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D2H6YLCPOXvZ",
        "outputId": "366a0f56-5d8e-4592-eccf-7035ebb5f5b2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.23.1-py3-none-any.whl (5.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.3 MB 4.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6 MB 72.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Collecting huggingface-hub<1.0,>=0.10.0\n",
            "  Downloading huggingface_hub-0.10.1-py3-none-any.whl (163 kB)\n",
            "\u001b[K     |████████████████████████████████| 163 kB 93.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.9.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.10.1 tokenizers-0.13.1 transformers-4.23.1\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.6.1-py3-none-any.whl (441 kB)\n",
            "\u001b[K     |████████████████████████████████| 441 kB 4.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Collecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.13.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.64.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.10.1)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (2022.8.2)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[K     |████████████████████████████████| 212 kB 63.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.5.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.21.6)\n",
            "Collecting multiprocess\n",
            "  Downloading multiprocess-0.70.13-py37-none-any.whl (115 kB)\n",
            "\u001b[K     |████████████████████████████████| 115 kB 69.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (22.1.0)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (6.0.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.1.1)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.1.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.8.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.8.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.9)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2022.9.24)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 70.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.9.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.4)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: urllib3, xxhash, responses, multiprocess, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "Successfully installed datasets-2.6.1 multiprocess-0.70.13 responses-0.18.0 urllib3-1.25.11 xxhash-3.1.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (1.21.6)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (1.3.5)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (1.21.6)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas) (2022.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas) (1.15.0)\n",
            "Archive:  data_and_models.zip\n",
            "   creating: data_and_models/\n",
            "  inflating: __MACOSX/._data_and_models  \n",
            "  inflating: data_and_models/logistic_model_8.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._logistic_model_8.pkl  \n",
            "  inflating: data_and_models/tfidf_44.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._tfidf_44.pkl  \n",
            "  inflating: data_and_models/tfidf_8.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._tfidf_8.pkl  \n",
            "  inflating: data_and_models/target_corpus.csv  \n",
            "  inflating: __MACOSX/data_and_models/._target_corpus.csv  \n",
            "  inflating: data_and_models/logistic_model_44.pkl  \n",
            "  inflating: __MACOSX/data_and_models/._logistic_model_44.pkl  \n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "!pip install datasets\n",
        "!pip install --upgrade --no-cache-dir gdown==4.5.4\n",
        "\n",
        "!gdown 18oZZ4jqRK-uF-Nz6ftRdgNjKix88hrnO\n",
        "!unzip data_and_models.zip && rm data_and_models.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "b7f18ad11a8442ef9205548349902354",
            "c9bba6f6c92b4613b221a91dea71e0dd",
            "6ed39e1f8a514ccda4a4044800dca953",
            "9e15b71d828349d58f9dec5d1651a297",
            "7f1c7d2c28154350923f1a04dd695703",
            "774ab9c7bd73479080002efd1bbf3309",
            "2429b841ab804e53ba67de76f71de0b2",
            "e48f3ef9534a43e381e739d720cb710a",
            "e98cb0705598456ba323075a7e5bf7aa",
            "7367637d1dde4ae2b8b7dfbf37837b6e",
            "b1536ac70f014899bce6eb6f8e422180",
            "b8b587710fb44031bc324db9d16bacee",
            "a9e83d047fbd4c6ba1066af666ad6c57",
            "156b8a7820174721996ba9b461eec49e",
            "664621768bf4486bbe45ed498dfca94d",
            "f9cf852e4f3d4dbb9689022aae1f82f2",
            "72fbb58bcace4f51bd2e588d6f2be935",
            "93195e1d3ff84050a34cc1951c958d19",
            "af8e9ad840584dcda834aa53eee90e6c",
            "0b8dac8f9aad462cbe61e2422ce40433",
            "32ff11be56ad4ee9bb8c7444bf91cc68",
            "6e5a26c529e04e0f84a0bb5f733bcd9a",
            "f114d148f87e4868a95c99134a84de1e",
            "42e49b98c19a429dbd6173ff2c918eca",
            "85adf013ab824127bad19389a4f52a79",
            "8cea3f00185745c9939a3f4fd095bd42",
            "8b83eec41773448790bdd211e45ea0d7",
            "8293a4bb14e54d15a6b52b687f2597d1",
            "54dd861c89f64b9889fea83529468a05",
            "c3e6b0efc447404e8e82cb62715241bc",
            "ef3d29f71b3448349d1c9c27fa861605",
            "061955c1f16c4ce686f701e9fac28289",
            "5337552047ce4ee4aa4665a423f73c64",
            "f6ae5e03aa78411fb6d41b07a7362f76",
            "c763a4ef5000477d89bceb45a4a63fd5",
            "fbbf095c019d4cf88d42f7da6439e2aa",
            "069f7ce845fb4237b871a18c91361bc7",
            "9e4f9593bc6e40ad842977bf749a351a",
            "a5600064a04a4a25bf0ac10ea1d171ec",
            "5aef8cd137724c8b8ec9b85902083a89",
            "70cbabd1bb0c4b3881b6534cfb53c40e",
            "304db1130fff445e9b4f1dddec818986",
            "d2ebfcda4b2540dd991a45edf239747b",
            "7ef3d4977a5d4164b890d961979f671d",
            "9fa62627472848e5848a207986743989",
            "c017eac956ee47e085d249e979eb4f98",
            "13d05ca809da4cc9baa4ca145db57ab4",
            "e9bbd7f3d4ea480bb3a143376d11af37",
            "280f43d15d214d51acc38625d73bcd8f",
            "7c0f94e595134eb49f40e42ce54ce0d7",
            "9719b51829374a898288401530067e18",
            "8e46eec488c54684a03d0cc3da4e347e",
            "0193028cc701443997c067c21ebd1345",
            "d31805d32f264b47b3f08a44ec5ecc79",
            "d40906ac92d94e2a8539d8f6f1d5469b",
            "53bf9e7f8aad4acfacb59d73710fe78a",
            "63e4f1e0c89b4b599215b0d034f70d92",
            "98ef45fde6c743e4a98779c7487c0c7d",
            "6192d566e54246be8970a30d69e656c5",
            "e4e9a3ca05fd428da9ce0be708474bed",
            "7321237bc340475da99151a5260905fe",
            "178d4c9dbd7c4c87be53cc9f459b8dfb",
            "b9a8ec4ece99485cb8fdfa5e21d6d0c9",
            "38024b92cbf04f4cae4d21fb78b78107",
            "755c3bdfb20948f68f24917d785d9af6",
            "551b889735784351976b1e86867b1f76"
          ]
        },
        "id": "H-LANn-hUlZh",
        "outputId": "1d827c3a-ca98-45f7-d5fd-29b571f2af16"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b7f18ad11a8442ef9205548349902354",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/899k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b8b587710fb44031bc324db9d16bacee",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f114d148f87e4868a95c99134a84de1e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f6ae5e03aa78411fb6d41b07a7362f76",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/481 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 36\n",
            "# classes in test 35\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9fa62627472848e5848a207986743989",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/501M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 25:48, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.430300</td>\n",
              "      <td>2.288778</td>\n",
              "      <td>0.417600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.917100</td>\n",
              "      <td>1.934722</td>\n",
              "      <td>0.516800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.582500</td>\n",
              "      <td>1.823273</td>\n",
              "      <td>0.542400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.362400</td>\n",
              "      <td>1.823743</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.972300</td>\n",
              "      <td>1.889241</td>\n",
              "      <td>0.523200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.848000</td>\n",
              "      <td>1.888951</td>\n",
              "      <td>0.534400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.573900</td>\n",
              "      <td>2.010903</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.409200</td>\n",
              "      <td>1.989290</td>\n",
              "      <td>0.553600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.317400</td>\n",
              "      <td>2.029555</td>\n",
              "      <td>0.563200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.230900</td>\n",
              "      <td>2.128701</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.196900</td>\n",
              "      <td>2.317828</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.136000</td>\n",
              "      <td>2.398464</td>\n",
              "      <td>0.540800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.049600</td>\n",
              "      <td>2.501200</td>\n",
              "      <td>0.545600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.136700</td>\n",
              "      <td>2.545634</td>\n",
              "      <td>0.552000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.057400</td>\n",
              "      <td>2.664694</td>\n",
              "      <td>0.544000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.082100</td>\n",
              "      <td>2.783216</td>\n",
              "      <td>0.531200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.014700</td>\n",
              "      <td>2.788378</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.063300</td>\n",
              "      <td>2.824948</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.027000</td>\n",
              "      <td>2.827253</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.009300</td>\n",
              "      <td>2.841008</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:12: FutureWarning: load_metric is deprecated and will be removed in the next major version of datasets. Use 'evaluate.load' instead, from the new library 🤗 Evaluate: https://huggingface.co/docs/evaluate\n",
            "  if sys.path[0] == '':\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "53bf9e7f8aad4acfacb59d73710fe78a",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading builder script:   0%|          | 0.00/1.65k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 1.8232734203338623).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 38\n",
            "# classes in test 36\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:07, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.440100</td>\n",
              "      <td>2.280676</td>\n",
              "      <td>0.435200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.108400</td>\n",
              "      <td>2.000825</td>\n",
              "      <td>0.473600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.461600</td>\n",
              "      <td>1.840411</td>\n",
              "      <td>0.512000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.201400</td>\n",
              "      <td>1.784630</td>\n",
              "      <td>0.528000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.943700</td>\n",
              "      <td>1.797061</td>\n",
              "      <td>0.547200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.615400</td>\n",
              "      <td>1.897894</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.641900</td>\n",
              "      <td>1.956028</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.229300</td>\n",
              "      <td>2.036530</td>\n",
              "      <td>0.544000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.324900</td>\n",
              "      <td>2.070697</td>\n",
              "      <td>0.537600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.199200</td>\n",
              "      <td>2.198223</td>\n",
              "      <td>0.531200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.191800</td>\n",
              "      <td>2.220739</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.091500</td>\n",
              "      <td>2.420457</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.148100</td>\n",
              "      <td>2.490360</td>\n",
              "      <td>0.540800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.057200</td>\n",
              "      <td>2.602995</td>\n",
              "      <td>0.544000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.028600</td>\n",
              "      <td>2.631632</td>\n",
              "      <td>0.545600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.048200</td>\n",
              "      <td>2.713568</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.013300</td>\n",
              "      <td>2.808079</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.036600</td>\n",
              "      <td>2.792651</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.038100</td>\n",
              "      <td>2.815949</td>\n",
              "      <td>0.534400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.008900</td>\n",
              "      <td>2.823409</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-732 (score: 1.7846299409866333).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1987: UserWarning: y_pred contains classes not in y_true\n",
            "  warnings.warn(\"y_pred contains classes not in y_true\")\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 41\n",
            "# classes in dev 38\n",
            "# classes in test 37\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:08, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.507300</td>\n",
              "      <td>2.389546</td>\n",
              "      <td>0.403200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.970200</td>\n",
              "      <td>2.135563</td>\n",
              "      <td>0.472000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.704300</td>\n",
              "      <td>1.997115</td>\n",
              "      <td>0.496000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.453000</td>\n",
              "      <td>2.035100</td>\n",
              "      <td>0.480000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.956900</td>\n",
              "      <td>2.062247</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.694500</td>\n",
              "      <td>2.155730</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.592900</td>\n",
              "      <td>2.221344</td>\n",
              "      <td>0.508800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.499300</td>\n",
              "      <td>2.261892</td>\n",
              "      <td>0.494400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.432800</td>\n",
              "      <td>2.324211</td>\n",
              "      <td>0.505600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.256400</td>\n",
              "      <td>2.473519</td>\n",
              "      <td>0.492800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.162800</td>\n",
              "      <td>2.528510</td>\n",
              "      <td>0.510400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.150600</td>\n",
              "      <td>2.699227</td>\n",
              "      <td>0.492800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.060500</td>\n",
              "      <td>2.807323</td>\n",
              "      <td>0.499200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.099500</td>\n",
              "      <td>2.924183</td>\n",
              "      <td>0.492800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.064200</td>\n",
              "      <td>3.009894</td>\n",
              "      <td>0.499200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.028200</td>\n",
              "      <td>3.092520</td>\n",
              "      <td>0.489600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.066600</td>\n",
              "      <td>3.182685</td>\n",
              "      <td>0.496000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.041900</td>\n",
              "      <td>3.218215</td>\n",
              "      <td>0.496000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.058300</td>\n",
              "      <td>3.211842</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.010000</td>\n",
              "      <td>3.221166</td>\n",
              "      <td>0.496000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 1.9971150159835815).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 35\n",
            "# classes in test 37\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:04, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.514900</td>\n",
              "      <td>2.500490</td>\n",
              "      <td>0.364800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2.062200</td>\n",
              "      <td>2.101844</td>\n",
              "      <td>0.464000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.538500</td>\n",
              "      <td>1.987228</td>\n",
              "      <td>0.472000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.217100</td>\n",
              "      <td>1.942860</td>\n",
              "      <td>0.502400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.870300</td>\n",
              "      <td>1.969932</td>\n",
              "      <td>0.489600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.730900</td>\n",
              "      <td>2.034905</td>\n",
              "      <td>0.489600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.683200</td>\n",
              "      <td>2.094531</td>\n",
              "      <td>0.499200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.339300</td>\n",
              "      <td>2.186576</td>\n",
              "      <td>0.488000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.295700</td>\n",
              "      <td>2.145506</td>\n",
              "      <td>0.518400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.235400</td>\n",
              "      <td>2.275206</td>\n",
              "      <td>0.492800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.127400</td>\n",
              "      <td>2.396797</td>\n",
              "      <td>0.505600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.302800</td>\n",
              "      <td>2.490853</td>\n",
              "      <td>0.500800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.092300</td>\n",
              "      <td>2.565019</td>\n",
              "      <td>0.513600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.062900</td>\n",
              "      <td>2.657266</td>\n",
              "      <td>0.520000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.052800</td>\n",
              "      <td>2.758803</td>\n",
              "      <td>0.523200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.066600</td>\n",
              "      <td>2.822627</td>\n",
              "      <td>0.513600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.041400</td>\n",
              "      <td>2.914744</td>\n",
              "      <td>0.513600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.017700</td>\n",
              "      <td>2.901634</td>\n",
              "      <td>0.521600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.020900</td>\n",
              "      <td>2.928650</td>\n",
              "      <td>0.520000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.022300</td>\n",
              "      <td>2.942898</td>\n",
              "      <td>0.513600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-732 (score: 1.9428601264953613).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1987: UserWarning: y_pred contains classes not in y_true\n",
            "  warnings.warn(\"y_pred contains classes not in y_true\")\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 42\n",
            "2915 625 625\n",
            "# classes in train 42\n",
            "# classes in dev 38\n",
            "# classes in test 35\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\",\n",
            "    \"8\": \"LABEL_8\",\n",
            "    \"9\": \"LABEL_9\",\n",
            "    \"10\": \"LABEL_10\",\n",
            "    \"11\": \"LABEL_11\",\n",
            "    \"12\": \"LABEL_12\",\n",
            "    \"13\": \"LABEL_13\",\n",
            "    \"14\": \"LABEL_14\",\n",
            "    \"15\": \"LABEL_15\",\n",
            "    \"16\": \"LABEL_16\",\n",
            "    \"17\": \"LABEL_17\",\n",
            "    \"18\": \"LABEL_18\",\n",
            "    \"19\": \"LABEL_19\",\n",
            "    \"20\": \"LABEL_20\",\n",
            "    \"21\": \"LABEL_21\",\n",
            "    \"22\": \"LABEL_22\",\n",
            "    \"23\": \"LABEL_23\",\n",
            "    \"24\": \"LABEL_24\",\n",
            "    \"25\": \"LABEL_25\",\n",
            "    \"26\": \"LABEL_26\",\n",
            "    \"27\": \"LABEL_27\",\n",
            "    \"28\": \"LABEL_28\",\n",
            "    \"29\": \"LABEL_29\",\n",
            "    \"30\": \"LABEL_30\",\n",
            "    \"31\": \"LABEL_31\",\n",
            "    \"32\": \"LABEL_32\",\n",
            "    \"33\": \"LABEL_33\",\n",
            "    \"34\": \"LABEL_34\",\n",
            "    \"35\": \"LABEL_35\",\n",
            "    \"36\": \"LABEL_36\",\n",
            "    \"37\": \"LABEL_37\",\n",
            "    \"38\": \"LABEL_38\",\n",
            "    \"39\": \"LABEL_39\",\n",
            "    \"40\": \"LABEL_40\",\n",
            "    \"41\": \"LABEL_41\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_10\": 10,\n",
            "    \"LABEL_11\": 11,\n",
            "    \"LABEL_12\": 12,\n",
            "    \"LABEL_13\": 13,\n",
            "    \"LABEL_14\": 14,\n",
            "    \"LABEL_15\": 15,\n",
            "    \"LABEL_16\": 16,\n",
            "    \"LABEL_17\": 17,\n",
            "    \"LABEL_18\": 18,\n",
            "    \"LABEL_19\": 19,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_20\": 20,\n",
            "    \"LABEL_21\": 21,\n",
            "    \"LABEL_22\": 22,\n",
            "    \"LABEL_23\": 23,\n",
            "    \"LABEL_24\": 24,\n",
            "    \"LABEL_25\": 25,\n",
            "    \"LABEL_26\": 26,\n",
            "    \"LABEL_27\": 27,\n",
            "    \"LABEL_28\": 28,\n",
            "    \"LABEL_29\": 29,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_30\": 30,\n",
            "    \"LABEL_31\": 31,\n",
            "    \"LABEL_32\": 32,\n",
            "    \"LABEL_33\": 33,\n",
            "    \"LABEL_34\": 34,\n",
            "    \"LABEL_35\": 35,\n",
            "    \"LABEL_36\": 36,\n",
            "    \"LABEL_37\": 37,\n",
            "    \"LABEL_38\": 38,\n",
            "    \"LABEL_39\": 39,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_40\": 40,\n",
            "    \"LABEL_41\": 41,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7,\n",
            "    \"LABEL_8\": 8,\n",
            "    \"LABEL_9\": 9\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:04, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.592000</td>\n",
              "      <td>2.406589</td>\n",
              "      <td>0.424000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.819100</td>\n",
              "      <td>2.115860</td>\n",
              "      <td>0.462400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>1.789500</td>\n",
              "      <td>1.973715</td>\n",
              "      <td>0.489600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>1.187000</td>\n",
              "      <td>1.987817</td>\n",
              "      <td>0.483200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>1.018000</td>\n",
              "      <td>2.026030</td>\n",
              "      <td>0.483200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.683700</td>\n",
              "      <td>2.087702</td>\n",
              "      <td>0.489600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.511400</td>\n",
              "      <td>2.160427</td>\n",
              "      <td>0.484800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.400500</td>\n",
              "      <td>2.265558</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.275900</td>\n",
              "      <td>2.363732</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.168700</td>\n",
              "      <td>2.525493</td>\n",
              "      <td>0.480000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.140500</td>\n",
              "      <td>2.654514</td>\n",
              "      <td>0.478400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.126400</td>\n",
              "      <td>2.768090</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.065700</td>\n",
              "      <td>2.897894</td>\n",
              "      <td>0.486400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.060300</td>\n",
              "      <td>2.967344</td>\n",
              "      <td>0.475200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.018900</td>\n",
              "      <td>3.023528</td>\n",
              "      <td>0.478400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.038000</td>\n",
              "      <td>3.081142</td>\n",
              "      <td>0.475200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.036500</td>\n",
              "      <td>3.214539</td>\n",
              "      <td>0.470400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.013900</td>\n",
              "      <td>3.237193</td>\n",
              "      <td>0.483200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.035000</td>\n",
              "      <td>3.215718</td>\n",
              "      <td>0.473600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.034400</td>\n",
              "      <td>3.229725</td>\n",
              "      <td>0.483200</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-549 (score: 1.9737151861190796).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1987: UserWarning: y_pred contains classes not in y_true\n",
            "  warnings.warn(\"y_pred contains classes not in y_true\")\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/base.py:338: UserWarning: Trying to unpickle estimator LogisticRegression from version 0.24.1 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations\n",
            "  UserWarning,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/base.py:338: UserWarning: Trying to unpickle estimator TfidfTransformer from version 0.24.1 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations\n",
            "  UserWarning,\n",
            "/usr/local/lib/python3.7/dist-packages/sklearn/base.py:338: UserWarning: Trying to unpickle estimator TfidfVectorizer from version 0.24.1 when using version 1.0.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
            "https://scikit-learn.org/stable/modules/model_persistence.html#security-maintainability-limitations\n",
            "  UserWarning,\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:11, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.319900</td>\n",
              "      <td>1.205588</td>\n",
              "      <td>0.577600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.978900</td>\n",
              "      <td>1.116255</td>\n",
              "      <td>0.619200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.816300</td>\n",
              "      <td>1.130859</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.821600</td>\n",
              "      <td>1.280719</td>\n",
              "      <td>0.632000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.496800</td>\n",
              "      <td>1.309494</td>\n",
              "      <td>0.628800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.146600</td>\n",
              "      <td>1.554289</td>\n",
              "      <td>0.609600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.165500</td>\n",
              "      <td>1.672504</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.178600</td>\n",
              "      <td>1.918652</td>\n",
              "      <td>0.628800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.038300</td>\n",
              "      <td>2.086765</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.116000</td>\n",
              "      <td>2.294300</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.003500</td>\n",
              "      <td>2.416271</td>\n",
              "      <td>0.636800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.001500</td>\n",
              "      <td>2.489655</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.004700</td>\n",
              "      <td>2.526492</td>\n",
              "      <td>0.652800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.000900</td>\n",
              "      <td>2.617174</td>\n",
              "      <td>0.627200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.019600</td>\n",
              "      <td>2.713682</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>2.661806</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.003200</td>\n",
              "      <td>2.688572</td>\n",
              "      <td>0.652800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.001600</td>\n",
              "      <td>2.701345</td>\n",
              "      <td>0.652800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.690373</td>\n",
              "      <td>0.654400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.695814</td>\n",
              "      <td>0.657600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.1162549257278442).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:00, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.139900</td>\n",
              "      <td>1.186527</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.909400</td>\n",
              "      <td>1.059664</td>\n",
              "      <td>0.616000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.685600</td>\n",
              "      <td>1.133391</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.607300</td>\n",
              "      <td>1.208220</td>\n",
              "      <td>0.635200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.316700</td>\n",
              "      <td>1.424255</td>\n",
              "      <td>0.619200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.182600</td>\n",
              "      <td>1.555377</td>\n",
              "      <td>0.632000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.190700</td>\n",
              "      <td>1.705964</td>\n",
              "      <td>0.624000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.098000</td>\n",
              "      <td>1.995148</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.007200</td>\n",
              "      <td>2.262979</td>\n",
              "      <td>0.617600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.013500</td>\n",
              "      <td>2.442546</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.002400</td>\n",
              "      <td>2.413244</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.002000</td>\n",
              "      <td>2.510874</td>\n",
              "      <td>0.640000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.002000</td>\n",
              "      <td>2.466660</td>\n",
              "      <td>0.657600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.001000</td>\n",
              "      <td>2.643223</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.001400</td>\n",
              "      <td>2.766240</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.011600</td>\n",
              "      <td>2.755085</td>\n",
              "      <td>0.627200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>2.788893</td>\n",
              "      <td>0.636800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.808046</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.743892</td>\n",
              "      <td>0.644800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000400</td>\n",
              "      <td>2.778039</td>\n",
              "      <td>0.641600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.0596638917922974).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 26:02, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.268100</td>\n",
              "      <td>1.286879</td>\n",
              "      <td>0.548800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.961000</td>\n",
              "      <td>1.170761</td>\n",
              "      <td>0.590400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.691900</td>\n",
              "      <td>1.191083</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.642400</td>\n",
              "      <td>1.341316</td>\n",
              "      <td>0.622400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.312900</td>\n",
              "      <td>1.541527</td>\n",
              "      <td>0.624000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.284600</td>\n",
              "      <td>1.650567</td>\n",
              "      <td>0.598400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.248600</td>\n",
              "      <td>1.889238</td>\n",
              "      <td>0.606400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.123300</td>\n",
              "      <td>2.147868</td>\n",
              "      <td>0.600000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.161900</td>\n",
              "      <td>2.279927</td>\n",
              "      <td>0.612800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.043300</td>\n",
              "      <td>2.516886</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.015800</td>\n",
              "      <td>2.671912</td>\n",
              "      <td>0.609600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.081300</td>\n",
              "      <td>2.677503</td>\n",
              "      <td>0.620800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.013100</td>\n",
              "      <td>2.728430</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.001700</td>\n",
              "      <td>2.875647</td>\n",
              "      <td>0.601600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.041100</td>\n",
              "      <td>2.786842</td>\n",
              "      <td>0.624000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.004300</td>\n",
              "      <td>3.022979</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>2.927167</td>\n",
              "      <td>0.616000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.001500</td>\n",
              "      <td>3.022665</td>\n",
              "      <td>0.616000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>2.997184</td>\n",
              "      <td>0.627200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000700</td>\n",
              "      <td>2.997494</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.1707608699798584).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 25:54, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.371100</td>\n",
              "      <td>1.301919</td>\n",
              "      <td>0.532800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.081300</td>\n",
              "      <td>1.196107</td>\n",
              "      <td>0.596800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.692400</td>\n",
              "      <td>1.225822</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.535300</td>\n",
              "      <td>1.311677</td>\n",
              "      <td>0.598400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.348000</td>\n",
              "      <td>1.458576</td>\n",
              "      <td>0.598400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.166300</td>\n",
              "      <td>1.601807</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.159400</td>\n",
              "      <td>1.825113</td>\n",
              "      <td>0.596800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.139200</td>\n",
              "      <td>2.099366</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.097400</td>\n",
              "      <td>2.326327</td>\n",
              "      <td>0.582400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.011300</td>\n",
              "      <td>2.486365</td>\n",
              "      <td>0.590400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.061800</td>\n",
              "      <td>2.600080</td>\n",
              "      <td>0.598400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.116500</td>\n",
              "      <td>2.727640</td>\n",
              "      <td>0.596800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.001800</td>\n",
              "      <td>2.828963</td>\n",
              "      <td>0.596800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.000900</td>\n",
              "      <td>2.878491</td>\n",
              "      <td>0.593600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.032900</td>\n",
              "      <td>2.924203</td>\n",
              "      <td>0.609600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.021500</td>\n",
              "      <td>2.962484</td>\n",
              "      <td>0.611200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000900</td>\n",
              "      <td>3.000527</td>\n",
              "      <td>0.606400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>3.034651</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.018000</td>\n",
              "      <td>3.044721</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>3.053281</td>\n",
              "      <td>0.608000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.196107268333435).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "# classes 8\n",
            "2915 625 625\n",
            "# classes in train 8\n",
            "# classes in dev 8\n",
            "# classes in test 8\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/config.json\n",
            "Model config RobertaConfig {\n",
            "  \"architectures\": [\n",
            "    \"RobertaForMaskedLM\"\n",
            "  ],\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"bos_token_id\": 0,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"eos_token_id\": 2,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"id2label\": {\n",
            "    \"0\": \"LABEL_0\",\n",
            "    \"1\": \"LABEL_1\",\n",
            "    \"2\": \"LABEL_2\",\n",
            "    \"3\": \"LABEL_3\",\n",
            "    \"4\": \"LABEL_4\",\n",
            "    \"5\": \"LABEL_5\",\n",
            "    \"6\": \"LABEL_6\",\n",
            "    \"7\": \"LABEL_7\"\n",
            "  },\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"label2id\": {\n",
            "    \"LABEL_0\": 0,\n",
            "    \"LABEL_1\": 1,\n",
            "    \"LABEL_2\": 2,\n",
            "    \"LABEL_3\": 3,\n",
            "    \"LABEL_4\": 4,\n",
            "    \"LABEL_5\": 5,\n",
            "    \"LABEL_6\": 6,\n",
            "    \"LABEL_7\": 7\n",
            "  },\n",
            "  \"layer_norm_eps\": 1e-05,\n",
            "  \"max_position_embeddings\": 514,\n",
            "  \"model_type\": \"roberta\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 1,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.23.1\",\n",
            "  \"type_vocab_size\": 1,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50265\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--roberta-base/snapshots/ff46155979338ff8063cdad90908b498ab91b181/pytorch_model.bin\n",
            "Some weights of the model checkpoint at roberta-base were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'roberta.pooler.dense.weight', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at roberta-base and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.weight', 'classifier.out_proj.bias', 'classifier.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  FutureWarning,\n",
            "***** Running training *****\n",
            "  Num examples = 2915\n",
            "  Num Epochs = 20\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 3660\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3660' max='3660' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [3660/3660 25:53, Epoch 20/20]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.292900</td>\n",
              "      <td>1.187130</td>\n",
              "      <td>0.603200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>1.051200</td>\n",
              "      <td>1.085045</td>\n",
              "      <td>0.633600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.789500</td>\n",
              "      <td>1.104119</td>\n",
              "      <td>0.632000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.691700</td>\n",
              "      <td>1.240097</td>\n",
              "      <td>0.606400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>0.346900</td>\n",
              "      <td>1.456409</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>6</td>\n",
              "      <td>0.258400</td>\n",
              "      <td>1.631499</td>\n",
              "      <td>0.635200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>7</td>\n",
              "      <td>0.291700</td>\n",
              "      <td>1.943097</td>\n",
              "      <td>0.614400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>8</td>\n",
              "      <td>0.063100</td>\n",
              "      <td>2.069870</td>\n",
              "      <td>0.630400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>9</td>\n",
              "      <td>0.150900</td>\n",
              "      <td>2.323310</td>\n",
              "      <td>0.609600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>0.007000</td>\n",
              "      <td>2.446229</td>\n",
              "      <td>0.612800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>11</td>\n",
              "      <td>0.005500</td>\n",
              "      <td>2.520805</td>\n",
              "      <td>0.622400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>12</td>\n",
              "      <td>0.001500</td>\n",
              "      <td>2.700134</td>\n",
              "      <td>0.625600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>13</td>\n",
              "      <td>0.000800</td>\n",
              "      <td>2.687340</td>\n",
              "      <td>0.620800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>14</td>\n",
              "      <td>0.001300</td>\n",
              "      <td>2.867251</td>\n",
              "      <td>0.601600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>15</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>2.868524</td>\n",
              "      <td>0.620800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>16</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.934491</td>\n",
              "      <td>0.620800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>17</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.981354</td>\n",
              "      <td>0.619200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>18</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>3.003122</td>\n",
              "      <td>0.616000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>19</td>\n",
              "      <td>0.000500</td>\n",
              "      <td>2.999850</td>\n",
              "      <td>0.616000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>0.000600</td>\n",
              "      <td>2.998413</td>\n",
              "      <td>0.612800</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-183\n",
            "Configuration saved in ./results/checkpoint-183/config.json\n",
            "Model weights saved in ./results/checkpoint-183/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-366\n",
            "Configuration saved in ./results/checkpoint-366/config.json\n",
            "Model weights saved in ./results/checkpoint-366/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-549\n",
            "Configuration saved in ./results/checkpoint-549/config.json\n",
            "Model weights saved in ./results/checkpoint-549/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-732\n",
            "Configuration saved in ./results/checkpoint-732/config.json\n",
            "Model weights saved in ./results/checkpoint-732/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-915\n",
            "Configuration saved in ./results/checkpoint-915/config.json\n",
            "Model weights saved in ./results/checkpoint-915/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1098\n",
            "Configuration saved in ./results/checkpoint-1098/config.json\n",
            "Model weights saved in ./results/checkpoint-1098/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1281\n",
            "Configuration saved in ./results/checkpoint-1281/config.json\n",
            "Model weights saved in ./results/checkpoint-1281/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1464\n",
            "Configuration saved in ./results/checkpoint-1464/config.json\n",
            "Model weights saved in ./results/checkpoint-1464/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1647\n",
            "Configuration saved in ./results/checkpoint-1647/config.json\n",
            "Model weights saved in ./results/checkpoint-1647/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-1830\n",
            "Configuration saved in ./results/checkpoint-1830/config.json\n",
            "Model weights saved in ./results/checkpoint-1830/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2013\n",
            "Configuration saved in ./results/checkpoint-2013/config.json\n",
            "Model weights saved in ./results/checkpoint-2013/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2196\n",
            "Configuration saved in ./results/checkpoint-2196/config.json\n",
            "Model weights saved in ./results/checkpoint-2196/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2379\n",
            "Configuration saved in ./results/checkpoint-2379/config.json\n",
            "Model weights saved in ./results/checkpoint-2379/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2562\n",
            "Configuration saved in ./results/checkpoint-2562/config.json\n",
            "Model weights saved in ./results/checkpoint-2562/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2745\n",
            "Configuration saved in ./results/checkpoint-2745/config.json\n",
            "Model weights saved in ./results/checkpoint-2745/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-2928\n",
            "Configuration saved in ./results/checkpoint-2928/config.json\n",
            "Model weights saved in ./results/checkpoint-2928/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3111\n",
            "Configuration saved in ./results/checkpoint-3111/config.json\n",
            "Model weights saved in ./results/checkpoint-3111/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3294\n",
            "Configuration saved in ./results/checkpoint-3294/config.json\n",
            "Model weights saved in ./results/checkpoint-3294/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3477\n",
            "Configuration saved in ./results/checkpoint-3477/config.json\n",
            "Model weights saved in ./results/checkpoint-3477/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n",
            "Saving model checkpoint to ./results/checkpoint-3660\n",
            "Configuration saved in ./results/checkpoint-3660/config.json\n",
            "Model weights saved in ./results/checkpoint-3660/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "Loading best model from ./results/checkpoint-366 (score: 1.085044503211975).\n",
            "***** Running Prediction *****\n",
            "  Num examples = 625\n",
            "  Batch size = 64\n"
          ]
        },
        {
          "data": {
            "text/html": [],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import random\n",
        "import time\n",
        "\n",
        "from datasets import load_metric\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import pickle\n",
        "\n",
        "\n",
        "def compute_metrics(eval_preds):\n",
        "    metric = load_metric(\"accuracy\")\n",
        "    logits, labels = eval_preds\n",
        "    predictions = np.argmax(logits, axis=-1)\n",
        "    return metric.compute(predictions=predictions, references=labels)\n",
        "\n",
        "start = time.time()\n",
        "directory = \"./data_and_models/\"\n",
        "all_df = pd.read_csv(directory+\"target_corpus.csv\")\n",
        "\n",
        "import torch\n",
        "class PSCDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "import sklearn\n",
        "def top_k_accuracy(top_k, predictions, labels):\n",
        "  assert len(predictions) == len(labels)\n",
        "  total = 0\n",
        "  correct = 0\n",
        "  for i in range(len(predictions)):\n",
        "    total += 1\n",
        "    prediction = []\n",
        "    for j, k in enumerate(predictions[i]):\n",
        "      prediction.append([j, k]) # k is the value\n",
        "    prediction.sort(key = lambda x: -x[1])\n",
        "    for j, _ in prediction[:top_k]:\n",
        "      if j == labels[i]:\n",
        "        correct += 1\n",
        "        break\n",
        "  return correct/total\n",
        "\n",
        "import transformers\n",
        "from transformers import RobertaTokenizerFast\n",
        "from transformers import RobertaForSequenceClassification, TrainingArguments, Trainer\n",
        "tokenizer = RobertaTokenizerFast.from_pretrained('roberta-base')\n",
        "\n",
        "\n",
        "metrics = [\"Top-1 accuracy/F1 micro\", \"Top-3 accuracy\", \"Top-5 accuracy\", \"Balanced accuracy\", \"F1 macro\"]\n",
        "seeds = [11, 12, 13, 14, 15]\n",
        "epochs = 20\n",
        "\n",
        "tasks = {\n",
        "    \"44\": {\n",
        "        \"number_of_labels\": 42,\n",
        "         \"label_column\": 1,\n",
        "    },\n",
        "    \"8\": {\n",
        "        \"number_of_labels\": 8,\n",
        "        \"label_column\": 2,\n",
        "    }\n",
        "}\n",
        "\n",
        "def compute(task):\n",
        "  t1, t3, t5, ba, f1 = [],[],[],[],[]\n",
        "  baseline_t1, baseline_t3, baseline_t5, baseline_ba, baseline_f1 = [], [], [], [], []\n",
        "\n",
        "  ###### load the cross-domain classifier\n",
        "  with open(directory + \"logistic_model_\" + task + \".pkl\", \"rb\") as doc:\n",
        "          model = pickle.load(doc)\n",
        "  with open(directory + \"tfidf_\" + task + \".pkl\", \"rb\") as doc:\n",
        "          cf_tokenizer = pickle.load(doc)\n",
        "  class_mapper = {}\n",
        "  class_reverse_mapper = {}\n",
        "  for i, topic in enumerate(model.classes_):\n",
        "      class_mapper[topic.replace(\" \", \".\").replace(\"-\", \".\")] = i\n",
        "      class_reverse_mapper[i] = topic\n",
        "\n",
        "  for seed in seeds:\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    random.seed(seed)\n",
        "\n",
        "    import csv\n",
        "    from sklearn.model_selection import train_test_split\n",
        "\n",
        "    index = -1\n",
        "    classes = {}\n",
        "    texts = []\n",
        "    labels = []\n",
        "    lm_reverse_mapper = {}\n",
        "    with open(directory + \"target_corpus.csv\") as doc:\n",
        "      reader = csv.reader(doc)\n",
        "      next(reader)\n",
        "      for row in reader:\n",
        "        topic = row[tasks[task][\"label_column\"]]\n",
        "        if topic not in classes:\n",
        "          index += 1\n",
        "          classes[topic] = index\n",
        "          lm_reverse_mapper[index] = topic.capitalize()\n",
        "        labels.append(classes[topic])\n",
        "        texts.append(row[0])\n",
        "    print(\"# classes\", len(classes))\n",
        "    X_train, X_test, y_train, y_test = train_test_split(texts, labels, test_size=625, random_state=seed)\n",
        "    X_train, X_dev, y_train, y_dev = train_test_split(X_train, y_train, test_size=625, random_state=seed)\n",
        "    print(len(X_train), len(X_dev), len(X_test))\n",
        "    print(\"# classes in train\", len(set(y_train)))\n",
        "    print(\"# classes in dev\", len(set(y_dev)))\n",
        "    print(\"# classes in test\", len(set(y_test)))\n",
        "\n",
        "    mlength = 512\n",
        "    train_encodings = tokenizer(X_train, truncation=True, padding=True, max_length=mlength)\n",
        "    dev_encodings = tokenizer(X_dev, truncation=True, padding=True, max_length = mlength)\n",
        "    test_encodings = tokenizer(X_test, truncation=True, padding=True, max_length= mlength)\n",
        "\n",
        "\n",
        "    train_dataset = PSCDataset(train_encodings, y_train)\n",
        "    dev_dataset = PSCDataset(dev_encodings, y_dev)\n",
        "    test_dataset = PSCDataset(test_encodings, y_test)\n",
        "\n",
        "    training_args = TrainingArguments(\n",
        "        output_dir=\"./results\",          # output directory\n",
        "        num_train_epochs=epochs,         # total number of training epochs\n",
        "        per_device_train_batch_size=16,  # batch size per device during training\n",
        "        per_device_eval_batch_size=64,   # batch size for evaluation\n",
        "        warmup_steps=0,                  # number of warmup steps for learning rate scheduler\n",
        "        weight_decay=0.01,               # strength of weight decay\n",
        "        logging_dir='./logs',            # directory for storing logs\n",
        "        logging_steps=10,\n",
        "        learning_rate = 2e-5,\n",
        "        save_strategy= \"epoch\",\n",
        "        evaluation_strategy=\"epoch\",\n",
        "        load_best_model_at_end= True,\n",
        "        seed = seed, \n",
        "    )\n",
        "\n",
        "    def model_init():\n",
        "        return RobertaForSequenceClassification.from_pretrained(\"roberta-base\", num_labels=tasks[task][\"number_of_labels\"])\n",
        "    trainer = Trainer(\n",
        "        model_init=model_init,               # the instantiated 🤗 Transformers model to be trained\n",
        "        args=training_args,                  # training arguments, defined above\n",
        "        train_dataset=train_dataset,         # training dataset\n",
        "        eval_dataset=dev_dataset,            # evaluation dataset\n",
        "        compute_metrics=compute_metrics,     # compute_metrics\n",
        "        )\n",
        "\n",
        "    trainer.train()\n",
        "    predictions = trainer.predict(test_dataset)\n",
        "    preds = np.argmax(predictions.predictions, axis=-1)\n",
        "\n",
        "    t1.append(top_k_accuracy(1, predictions.predictions, test_dataset.labels))\n",
        "    t3.append(top_k_accuracy(3, predictions.predictions, test_dataset.labels))\n",
        "    t5.append(top_k_accuracy(5, predictions.predictions, test_dataset.labels))\n",
        "    ba.append(sklearn.metrics.balanced_accuracy_score(test_dataset.labels, preds))\n",
        "    f1.append(sklearn.metrics.f1_score(test_dataset.labels, preds, average = \"macro\"))\n",
        "\n",
        "    df = all_df[all_df[\"text\"].isin(X_test)]\n",
        "    X = df['text']\n",
        "    Y = list(df[\"topic_\"+ task].transform(lambda x: class_mapper[x]))\n",
        "\n",
        "    Xtfidf = cf_tokenizer.transform(X)\n",
        "\n",
        "    preds = model.predict(Xtfidf)\n",
        "    preds = [class_mapper[topic.replace(\" \", \".\").replace(\"-\", \".\")] for topic in preds]\n",
        "    policy_probs = model.predict_proba(Xtfidf)\n",
        "    \n",
        "    baseline_t1.append(top_k_accuracy(1, policy_probs, Y))\n",
        "    baseline_t3.append(top_k_accuracy(3, policy_probs, Y))\n",
        "    baseline_t5.append(top_k_accuracy(5, policy_probs, Y))\n",
        "    baseline_ba.append(sklearn.metrics.balanced_accuracy_score(Y, preds))\n",
        "    baseline_f1.append(sklearn.metrics.f1_score(Y, preds, average = \"macro\"))\n",
        "\n",
        "  result = {}\n",
        "  for metric, baseline, experiment in zip(metrics, [baseline_t1, baseline_t3, baseline_t5, baseline_ba, baseline_f1], [t1, t3, t5, ba, f1]):\n",
        "    result[metric] = [np.mean(baseline), np.std(baseline), np.mean(experiment), np.std(baseline)]\n",
        "  return result\n",
        "\n",
        "results = {}\n",
        "for task in tasks:\n",
        "  result = compute(task)\n",
        "  results[task] = result\n",
        "\n",
        "np.save( directory + \"table_1_results.npy\", results)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "lLXYkZy1X7dh",
        "outputId": "d8a03322-3c25-4da7-b818-4fdda76ee1cc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Top-1 accuracy/F1 micro & 0.414 (0.009) &  \\textbf{0.527 (0.009)} & 0.515 (0.006) &  \\textbf{0.631 (0.006)}\\\\\n",
            "Top-3 accuracy & 0.656 (0.008) &  \\textbf{0.744 (0.008)} & 0.819 (0.003) &  \\textbf{0.904 (0.003)}\\\\\n",
            "Top-5 accuracy & 0.752 (0.004) &  \\textbf{0.828 (0.004)} & 0.921 (0.008) &  \\textbf{0.969 (0.008)}\\\\\n",
            "Balanced accuracy & 0.309 (0.030) &  \\textbf{0.357 (0.030)} & 0.454 (0.014) &  \\textbf{0.580 (0.014)}\\\\\n",
            "F1 macro & 0.294 (0.025) &  \\textbf{0.328 (0.025)} & 0.449 (0.014) &  \\textbf{0.574 (0.014)}\\\\\n"
          ]
        }
      ],
      "source": [
        "def preprocess_result(result):\n",
        "  output = []\n",
        "  outperform = result[2] > result[0]\n",
        "  for i, j in enumerate(result):\n",
        "    j = str(round(j, 3))\n",
        "    if len(j) < 5:\n",
        "      j += \"0\" * (5-len(j))\n",
        "    if i % 2 == 1: # standard deviation\n",
        "      j = \"(\" + j + \")\"\n",
        "      if i == 1:\n",
        "        j += \" & \"\n",
        "    if outperform:\n",
        "      if i == 2:\n",
        "        j = \"\\\\textbf{\" + j\n",
        "      if i == 3:\n",
        "        j += \"}\"\n",
        "    else:\n",
        "      if i == 0:\n",
        "        j = \"\\\\textbf{\" + j\n",
        "      if i == 1:\n",
        "        j += \"}\"\n",
        "    output.append(j)\n",
        "  return \" \".join(output)\n",
        "\n",
        "for metric in metrics:\n",
        "  output = [metric]\n",
        "  for task in tasks:\n",
        "    output.append(preprocess_result(results[task][metric]))\n",
        "  print(\" & \".join(output) + \"\\\\\\\\\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "Yrtzj-vwK9h1",
        "outputId": "c60036b5-d6e1-4529-c722-fdf121a61985"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The program took 262.0 minutes in total.\n"
          ]
        }
      ],
      "source": [
        "end = time.time()\n",
        "print(f\"The program took {(end - start) // 60} minutes in total.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "DPOufc4bLO1o"
      },
      "outputs": [],
      "source": [
        "from google.colab import runtime\n",
        "runtime.unassign()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0193028cc701443997c067c21ebd1345": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "061955c1f16c4ce686f701e9fac28289": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "069f7ce845fb4237b871a18c91361bc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d2ebfcda4b2540dd991a45edf239747b",
            "placeholder": "​",
            "style": "IPY_MODEL_7ef3d4977a5d4164b890d961979f671d",
            "value": " 481/481 [00:00&lt;00:00, 19.4kB/s]"
          }
        },
        "0b8dac8f9aad462cbe61e2422ce40433": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "13d05ca809da4cc9baa4ca145db57ab4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8e46eec488c54684a03d0cc3da4e347e",
            "max": 501200538,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0193028cc701443997c067c21ebd1345",
            "value": 501200538
          }
        },
        "156b8a7820174721996ba9b461eec49e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_af8e9ad840584dcda834aa53eee90e6c",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0b8dac8f9aad462cbe61e2422ce40433",
            "value": 456318
          }
        },
        "178d4c9dbd7c4c87be53cc9f459b8dfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2429b841ab804e53ba67de76f71de0b2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "280f43d15d214d51acc38625d73bcd8f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "304db1130fff445e9b4f1dddec818986": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "32ff11be56ad4ee9bb8c7444bf91cc68": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "38024b92cbf04f4cae4d21fb78b78107": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "42e49b98c19a429dbd6173ff2c918eca": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8293a4bb14e54d15a6b52b687f2597d1",
            "placeholder": "​",
            "style": "IPY_MODEL_54dd861c89f64b9889fea83529468a05",
            "value": "Downloading: 100%"
          }
        },
        "5337552047ce4ee4aa4665a423f73c64": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "53bf9e7f8aad4acfacb59d73710fe78a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_63e4f1e0c89b4b599215b0d034f70d92",
              "IPY_MODEL_98ef45fde6c743e4a98779c7487c0c7d",
              "IPY_MODEL_6192d566e54246be8970a30d69e656c5"
            ],
            "layout": "IPY_MODEL_e4e9a3ca05fd428da9ce0be708474bed"
          }
        },
        "54dd861c89f64b9889fea83529468a05": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "551b889735784351976b1e86867b1f76": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5aef8cd137724c8b8ec9b85902083a89": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6192d566e54246be8970a30d69e656c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_755c3bdfb20948f68f24917d785d9af6",
            "placeholder": "​",
            "style": "IPY_MODEL_551b889735784351976b1e86867b1f76",
            "value": " 4.21k/? [00:00&lt;00:00, 157kB/s]"
          }
        },
        "63e4f1e0c89b4b599215b0d034f70d92": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7321237bc340475da99151a5260905fe",
            "placeholder": "​",
            "style": "IPY_MODEL_178d4c9dbd7c4c87be53cc9f459b8dfb",
            "value": "Downloading builder script: "
          }
        },
        "664621768bf4486bbe45ed498dfca94d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_32ff11be56ad4ee9bb8c7444bf91cc68",
            "placeholder": "​",
            "style": "IPY_MODEL_6e5a26c529e04e0f84a0bb5f733bcd9a",
            "value": " 456k/456k [00:00&lt;00:00, 1.54MB/s]"
          }
        },
        "6e5a26c529e04e0f84a0bb5f733bcd9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6ed39e1f8a514ccda4a4044800dca953": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e48f3ef9534a43e381e739d720cb710a",
            "max": 898823,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e98cb0705598456ba323075a7e5bf7aa",
            "value": 898823
          }
        },
        "70cbabd1bb0c4b3881b6534cfb53c40e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "72fbb58bcace4f51bd2e588d6f2be935": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7321237bc340475da99151a5260905fe": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7367637d1dde4ae2b8b7dfbf37837b6e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "755c3bdfb20948f68f24917d785d9af6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "774ab9c7bd73479080002efd1bbf3309": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c0f94e595134eb49f40e42ce54ce0d7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7ef3d4977a5d4164b890d961979f671d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7f1c7d2c28154350923f1a04dd695703": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8293a4bb14e54d15a6b52b687f2597d1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "85adf013ab824127bad19389a4f52a79": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c3e6b0efc447404e8e82cb62715241bc",
            "max": 1355863,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ef3d29f71b3448349d1c9c27fa861605",
            "value": 1355863
          }
        },
        "8b83eec41773448790bdd211e45ea0d7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8cea3f00185745c9939a3f4fd095bd42": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_061955c1f16c4ce686f701e9fac28289",
            "placeholder": "​",
            "style": "IPY_MODEL_5337552047ce4ee4aa4665a423f73c64",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 7.12MB/s]"
          }
        },
        "8e46eec488c54684a03d0cc3da4e347e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "93195e1d3ff84050a34cc1951c958d19": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9719b51829374a898288401530067e18": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "98ef45fde6c743e4a98779c7487c0c7d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b9a8ec4ece99485cb8fdfa5e21d6d0c9",
            "max": 1652,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_38024b92cbf04f4cae4d21fb78b78107",
            "value": 1652
          }
        },
        "9e15b71d828349d58f9dec5d1651a297": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7367637d1dde4ae2b8b7dfbf37837b6e",
            "placeholder": "​",
            "style": "IPY_MODEL_b1536ac70f014899bce6eb6f8e422180",
            "value": " 899k/899k [00:00&lt;00:00, 1.66MB/s]"
          }
        },
        "9e4f9593bc6e40ad842977bf749a351a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9fa62627472848e5848a207986743989": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c017eac956ee47e085d249e979eb4f98",
              "IPY_MODEL_13d05ca809da4cc9baa4ca145db57ab4",
              "IPY_MODEL_e9bbd7f3d4ea480bb3a143376d11af37"
            ],
            "layout": "IPY_MODEL_280f43d15d214d51acc38625d73bcd8f"
          }
        },
        "a5600064a04a4a25bf0ac10ea1d171ec": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a9e83d047fbd4c6ba1066af666ad6c57": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_72fbb58bcace4f51bd2e588d6f2be935",
            "placeholder": "​",
            "style": "IPY_MODEL_93195e1d3ff84050a34cc1951c958d19",
            "value": "Downloading: 100%"
          }
        },
        "af8e9ad840584dcda834aa53eee90e6c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b1536ac70f014899bce6eb6f8e422180": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b7f18ad11a8442ef9205548349902354": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c9bba6f6c92b4613b221a91dea71e0dd",
              "IPY_MODEL_6ed39e1f8a514ccda4a4044800dca953",
              "IPY_MODEL_9e15b71d828349d58f9dec5d1651a297"
            ],
            "layout": "IPY_MODEL_7f1c7d2c28154350923f1a04dd695703"
          }
        },
        "b8b587710fb44031bc324db9d16bacee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a9e83d047fbd4c6ba1066af666ad6c57",
              "IPY_MODEL_156b8a7820174721996ba9b461eec49e",
              "IPY_MODEL_664621768bf4486bbe45ed498dfca94d"
            ],
            "layout": "IPY_MODEL_f9cf852e4f3d4dbb9689022aae1f82f2"
          }
        },
        "b9a8ec4ece99485cb8fdfa5e21d6d0c9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c017eac956ee47e085d249e979eb4f98": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7c0f94e595134eb49f40e42ce54ce0d7",
            "placeholder": "​",
            "style": "IPY_MODEL_9719b51829374a898288401530067e18",
            "value": "Downloading: 100%"
          }
        },
        "c3e6b0efc447404e8e82cb62715241bc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c763a4ef5000477d89bceb45a4a63fd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a5600064a04a4a25bf0ac10ea1d171ec",
            "placeholder": "​",
            "style": "IPY_MODEL_5aef8cd137724c8b8ec9b85902083a89",
            "value": "Downloading: 100%"
          }
        },
        "c9bba6f6c92b4613b221a91dea71e0dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_774ab9c7bd73479080002efd1bbf3309",
            "placeholder": "​",
            "style": "IPY_MODEL_2429b841ab804e53ba67de76f71de0b2",
            "value": "Downloading: 100%"
          }
        },
        "d2ebfcda4b2540dd991a45edf239747b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d31805d32f264b47b3f08a44ec5ecc79": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d40906ac92d94e2a8539d8f6f1d5469b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e48f3ef9534a43e381e739d720cb710a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e4e9a3ca05fd428da9ce0be708474bed": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e98cb0705598456ba323075a7e5bf7aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e9bbd7f3d4ea480bb3a143376d11af37": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d31805d32f264b47b3f08a44ec5ecc79",
            "placeholder": "​",
            "style": "IPY_MODEL_d40906ac92d94e2a8539d8f6f1d5469b",
            "value": " 501M/501M [00:08&lt;00:00, 60.2MB/s]"
          }
        },
        "ef3d29f71b3448349d1c9c27fa861605": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f114d148f87e4868a95c99134a84de1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_42e49b98c19a429dbd6173ff2c918eca",
              "IPY_MODEL_85adf013ab824127bad19389a4f52a79",
              "IPY_MODEL_8cea3f00185745c9939a3f4fd095bd42"
            ],
            "layout": "IPY_MODEL_8b83eec41773448790bdd211e45ea0d7"
          }
        },
        "f6ae5e03aa78411fb6d41b07a7362f76": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c763a4ef5000477d89bceb45a4a63fd5",
              "IPY_MODEL_fbbf095c019d4cf88d42f7da6439e2aa",
              "IPY_MODEL_069f7ce845fb4237b871a18c91361bc7"
            ],
            "layout": "IPY_MODEL_9e4f9593bc6e40ad842977bf749a351a"
          }
        },
        "f9cf852e4f3d4dbb9689022aae1f82f2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fbbf095c019d4cf88d42f7da6439e2aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_70cbabd1bb0c4b3881b6534cfb53c40e",
            "max": 481,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_304db1130fff445e9b4f1dddec818986",
            "value": 481
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}